
# import packages
import pandas as pd
import numpy as np
import warnings

import matplotlib.pyplot as plt
import seaborn as sns
from tqdm import tqdm

import re
from pandas.api.types import CategoricalDtype
from scipy import sparse
from sklearn.preprocessing import LabelEncoder

from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.model_selection import cross_val_score, train_test_split, StratifiedKFold
from sklearn.metrics import f1_score, accuracy_score
from sklearn.ensemble import VotingClassifier

from xgboost import XGBRegressor, XGBClassifier
from lightgbm import LGBMRegressor, LGBMClassifier, LGBMRanker, early_stopping
from catboost import CatBoostRegressor, CatBoostClassifier, Pool

from surprise import Dataset, Reader, accuracy, SVD, KNNBasic, CoClustering
from surprise.dataset import DatasetAutoFolds


warnings.filterwarnings(action='ignore')


users=pd.read_csv('users_2.0.csv')
books=pd.read_csv('books_2.0.csv')
ratings=pd.read_csv('data/train_ratings.csv')
submission_test=pd.read_csv('data/sample_submission.csv')

SEED=8746
dictionary=pd.concat([ratings,submission_test])
user2idx = {v:k for k,v in enumerate(dictionary['user_id'].unique())}
book2idx = {v:k for k,v in enumerate(dictionary['isbn'].unique())}


merge1 = ratings.merge(books, how='left', on='isbn')
dataset = merge1.merge(users, how='inner', on='user_id')
dataset.columns

Index(['user_id', 'isbn', 'rating', 'book_title', 'book_author',
       'year_of_publication', 'publisher', 'img_url', 'language', 'category',
       'summary', 'img_path', 'age', 'location_country'],
      dtype='object')


from pandas.api.types import CategoricalDtype
from scipy import sparse
ratings['iid'] = ratings['isbn'].map(book2idx)
ratings['uid'] = ratings['user_id'].map(user2idx)

size_uid = ratings["uid"].unique()
size_iid = ratings["iid"].unique()

ui_shape = (len(size_uid), len(size_iid))

user_cat = CategoricalDtype(categories=sorted(size_uid), ordered=True)
book_cat = CategoricalDtype(categories=sorted(size_iid), ordered=True)

user_index = ratings["uid"].astype(user_cat).cat.codes
book_index = ratings["iid"].astype(book_cat).cat.codes

interactions = sparse.coo_matrix((ratings["rating"], (user_index,book_index)), shape=ui_shape)


def check_sparsity(interactions:sparse.coo_matrix)->int:
  matrix_size = interactions.shape[0]*interactions.shape[1] # Number of possible interactions in the matrix
  num_purchases = len(interactions.nonzero()[0]) # Number of items interacted with
  sparsity = 100*(1 - (num_purchases/matrix_size))
  return round(sparsity,4)
#sparse 확인
print(f"matrix sparsity : {check_sparsity(interactions)}%")

matrix sparsity : 99.996%


def shuffle_data(interactions:sparse.coo_matrix, random_state:int=42)->tuple:
    random_state = np.random.RandomState(seed=random_state)

    interactions = interactions.tocoo()

    uids, iids, data = (interactions.row, interactions.col, interactions.data)

    shuffle_indices = np.arange(len(uids))
    random_state.shuffle(shuffle_indices)

    uids = uids[shuffle_indices]
    iids = iids[shuffle_indices]
    data = data[shuffle_indices]

    return uids, iids, data


def cutoff_by_user(uids:list, test_percentage:float=0.2):
    cutoff = int((1.0 - test_percentage) * len(uids))
    train_idx = slice(None, cutoff)
    test_idx = slice(cutoff, None)
    return train_idx, test_idx
def random_train_test_split(interactions, test_percentage=0.2, valid=None, random_state=42):
    uids, iids, data = shuffle_data(interactions, random_state)
    train_idx, test_idx = cutoff_by_user(uids, test_percentage)
    shape = interactions.shape

    train = sparse.coo_matrix(
        (data[train_idx], (uids[train_idx], iids[train_idx])),
        shape=shape,
        dtype=interactions.dtype,
    )
    test = sparse.coo_matrix(
        (data[test_idx], (uids[test_idx], iids[test_idx])),
        shape=shape,
        dtype=interactions.dtype,
    )
    return train, test
train, test = random_train_test_split(interactions,test_percentage=0.2,random_state=42)
train = train.tocsr()
test = test.tocsr()
train.shape, test.shape

((59803, 129777), (59803, 129777))


uids, iids, data = shuffle_data(interactions)
train_idx, test_idx = cutoff_by_user(uids)

train_df = pd.DataFrame({'uid':uids[train_idx], 'iid':iids[train_idx], 'ratings':data[train_idx]})
test_df = pd.DataFrame({'uid':uids[test_idx], 'iid':iids[test_idx], 'ratings':data[test_idx]})
test_df


users_df=users.copy()


users_df['uid'] = users_df['user_id'].map(user2idx)


users_df['age'] = users_df['age'].map(lambda x:x//10 if type(x)!=str else -1)
#연령별 카테고리화


#users_df.dropna(inplace=True)
users_df.fillna(0)


books_df = books.copy()
books_df['iid'] = books_df['isbn'].map(book2idx)
books_df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 149570 entries, 0 to 149569
Data columns (total 11 columns):
 #   Column               Non-Null Count   Dtype  
---  ------               --------------   -----  
 0   isbn                 149570 non-null  object 
 1   book_title           149570 non-null  object 
 2   book_author          149570 non-null  object 
 3   year_of_publication  149570 non-null  float64
 4   publisher            149570 non-null  int64  
 5   img_url              149570 non-null  object 
 6   language             149570 non-null  int64  
 7   category             130751 non-null  float64
 8   summary              82343 non-null   object 
 9   img_path             149570 non-null  object 
 10  iid                  149570 non-null  int64  
dtypes: float64(2), int64(3), object(6)
memory usage: 12.6+ MB


train_context = train_df.merge(users_df, on='uid', how='left').merge(books_df, on='iid', how='left')
test_context = test_df.merge(users_df, on='uid', how='left').merge(books_df, on='iid', how='left')


X_train = train_context.drop(['user_id', 'isbn', 'iid','uid', 'ratings'], axis=1)
y_train = train_context['ratings']
rating_train = train_context[['uid','iid','ratings']] 

X_test = test_context.drop(['user_id', 'isbn', 'iid','uid', 'ratings'], axis=1)
y_test = test_context['ratings']
rating_test = test_context[['uid','iid','ratings']]


drop_lst = ['book_title','book_author','img_url','img_path','summary']

X_train.drop(drop_lst,axis=1,inplace=True)
X_test.drop(drop_lst,axis=1,inplace=True)


def modify_range(rating):
  if rating < 0:
    return 0
  elif rating > 10:
    return 10
  else:
    return rating
def rmse(real, predict):
  pred = list(map(modify_range, predict))  
  pred = np.array(pred)
  return np.sqrt(np.mean((real-pred) ** 2))
def matrix_rmse(real_mat, predict_mat, test_ind=test_idx):
  cost = 0

  for i, ind in enumerate(test_ind):
    pred = predict_mat[ind]
    real = real_mat[ind]
    cost += pow(real - modify_range(pred), 2)
  return np.sqrt(cost/ len(test_ind))


X_train


from surprise import accuracy, Dataset, SVD
from surprise.model_selection import KFold
import optuna

trials=500
model_params={}
model_scores={}


#SVD

reader = Reader(rating_scale=(1, 10))
train_data = Dataset.load_from_df(train_df[['uid', 'iid', 'ratings']], reader)
test_data = Dataset.load_from_df(test_df[['uid', 'iid', 'ratings']], reader)
fold = DatasetAutoFolds(df = train_df[['uid', 'iid', 'ratings']], reader = reader)
trainset = fold.build_full_trainset()

def svd_objective(trial):
    params = {'n_factors':trial.suggest_int('n_factors', 50, 200),
          'n_epochs':trial.suggest_int('n_epochs', 10, 150),
          'lr_all':trial.suggest_loguniform("lr_all", 0.001, 0.1),
          'reg_all':trial.suggest_loguniform('reg_all', 0.01, 0.1),
          }

    
    fold = KFold(n_splits=5, shuffle=True, random_state=SEED)
    cv_scores = []


    svd = SVD(**params,random_state=SEED)
    svd.fit(trainset)
    uir_tuple = list(zip(test_df['uid'], test_df['iid'],test_df['ratings']))
    svd_test = svd.test(uir_tuple)
    svd_pred = list(map(lambda x:round(x.est), svd_test))
    score = rmse(y_test, svd_pred)

    cv_scores.append(score)
    print("SVD Classifier: ",score)
    return np.mean(cv_scores)

svd_study = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=SEED))
svd_study.optimize(svd_objective, n_trials=trials)

svd_best = svd_study.best_trial
svd_best_params = svd_best.params

model_scores['svd'] = svd_best.value
model_params['svd'] = svd_best_params

print('score: {0}, params: {1}'.format(svd_best.value, svd_best_params))

[I 2022-11-02 17:44:19,082] A new study created in memory with name: no-name-6e0ccfbc-2128-4b2b-99cf-0a17eeb49e48
[I 2022-11-02 17:44:45,055] Trial 0 finished with value: 2.2184221924763214 and parameters: {'n_factors': 93, 'n_epochs': 104, 'lr_all': 0.0012843500757831465, 'reg_all': 0.015461639312916838}. Best is trial 0 with value: 2.2184221924763214.

SVD Classifier:  2.2184221924763214

[I 2022-11-02 17:45:04,706] Trial 1 finished with value: 2.212323602731479 and parameters: {'n_factors': 150, 'n_epochs': 63, 'lr_all': 0.004464451983251425, 'reg_all': 0.06916649384161133}. Best is trial 0 with value: 2.2184221924763214.

SVD Classifier:  2.212323602731479

[I 2022-11-02 17:45:38,920] Trial 2 finished with value: 2.2696094320472935 and parameters: {'n_factors': 170, 'n_epochs': 105, 'lr_all': 0.09584449931600303, 'reg_all': 0.06855496343839224}. Best is trial 2 with value: 2.2696094320472935.

SVD Classifier:  2.2696094320472935

[I 2022-11-02 17:45:49,652] Trial 3 finished with value: 2.227425363961006 and parameters: {'n_factors': 76, 'n_epochs': 45, 'lr_all': 0.019127074813559148, 'reg_all': 0.014076726730908193}. Best is trial 2 with value: 2.2696094320472935.

SVD Classifier:  2.227425363961006

[I 2022-11-02 17:46:07,113] Trial 4 finished with value: 2.2171325156293693 and parameters: {'n_factors': 140, 'n_epochs': 56, 'lr_all': 0.012162731847853712, 'reg_all': 0.017792027825339566}. Best is trial 2 with value: 2.2696094320472935.

SVD Classifier:  2.2171325156293693


optuna.visualization.plot_param_importances(svd_study) # 파라미터 중요도 확인 그래프
optuna.visualization.plot_optimization_history(svd_study) # 최적화 과정 시각화


# LGBM_r
def lgbm_r_objective(trial):
    param = {
        'objective':'rmse',
        'boosting_type':'gbdt',
        'num_leaves': trial.suggest_int('num_leaves', 10, 150),
        'max_depth': trial.suggest_int('max_depth', 3, 20),
        'learning_rate': trial.suggest_loguniform("learning_rate", 0.01, 0.1),
        'n_estimators': trial.suggest_int('n_estimators', 50, 3000),
        'class_weight':trial.suggest_categorical('class_weight',[None,'balanced']),
        'min_child_samples': trial.suggest_int('min_child_samples', 2, 50),
        'min_child_weight': trial.suggest_uniform('min_child_weight', 0, 1),
        'reg_alpha':trial.suggest_float('reg_alpha',0,1),
        'reg_lambda':trial.suggest_float('reg_lambda',0,1)
    }


        
    
    fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)
    cv_scores = []
    for train_idx, val_idx in fold.split(X_train, y_train):
        X_tr, X_val = X_train.iloc[train_idx], X_train.iloc[val_idx]
        y_tr, y_val = y_train.iloc[train_idx], y_train.iloc[val_idx]

        model = LGBMRegressor(**param, n_jobs=-1, random_state=SEED)
        model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)], 
                  callbacks=[early_stopping(stopping_rounds=50, verbose=False)])
        y_pred = model.predict(X_val)
        score = rmse(y_val, y_pred)

        cv_scores.append(score)
        print("LGBM Regressor: ",score)
    return np.mean(cv_scores)

lgb_roc_study = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=42))
lgb_roc_study.optimize(lgbm_r_objective, n_trials=trials)

lgb_roc_best = lgb_roc_study.best_trial
lgb_roc_best_params = lgb_roc_best.params

model_scores['lgb_r'] = lgb_roc_best.value
model_params['lgb_r'] = lgb_roc_best_params

print('score: {0}, params: {1}'.format(lgb_roc_best.value, lgb_roc_best_params))

[I 2022-11-03 01:18:38,453] A new study created in memory with name: no-name-2169b889-fa45-4c18-bb74-d1c371b12dd7

LGBM Regressor:  2.3919800252600063
LGBM Regressor:  2.3891025760750066
LGBM Regressor:  2.3924288929779283
LGBM Regressor:  2.3916552072865467

[I 2022-11-03 01:19:08,834] Trial 0 finished with value: 2.391261520925382 and parameters: {'num_leaves': 62, 'max_depth': 20, 'learning_rate': 0.05395030966670229, 'n_estimators': 1816, 'class_weight': None, 'min_child_samples': 4, 'min_child_weight': 0.8661761457749352, 'reg_alpha': 0.6011150117432088, 'reg_lambda': 0.7080725777960455}. Best is trial 0 with value: 2.391261520925382.

LGBM Regressor:  2.3911409030274227
LGBM Regressor:  2.827852561907342
LGBM Regressor:  2.825472968380633
LGBM Regressor:  2.8291749111047486
LGBM Regressor:  2.8309261402894874

[I 2022-11-03 01:19:28,992] Trial 1 finished with value: 2.8278186084492374 and parameters: {'num_leaves': 12, 'max_depth': 20, 'learning_rate': 0.06798962421591129, 'n_estimators': 676, 'class_weight': 'balanced', 'min_child_samples': 16, 'min_child_weight': 0.5247564316322378, 'reg_alpha': 0.43194501864211576, 'reg_lambda': 0.2912291401980419}. Best is trial 1 with value: 2.8278186084492374.

LGBM Regressor:  2.8256664605639754
LGBM Regressor:  2.828097871380376
LGBM Regressor:  2.8265849273259716
LGBM Regressor:  2.8303654593275738
LGBM Regressor:  2.8318951994263113

[I 2022-11-03 01:20:15,350] Trial 2 finished with value: 2.8290344637500597 and parameters: {'num_leaves': 96, 'max_depth': 5, 'learning_rate': 0.019594972058679168, 'n_estimators': 1131, 'class_weight': 'balanced', 'min_child_samples': 11, 'min_child_weight': 0.5142344384136116, 'reg_alpha': 0.5924145688620425, 'reg_lambda': 0.046450412719997725}. Best is trial 2 with value: 2.8290344637500597.

LGBM Regressor:  2.8282288612900666
LGBM Regressor:  2.392944346604092
LGBM Regressor:  2.3898071775555008
LGBM Regressor:  2.3925103408525352
LGBM Regressor:  2.3919302197286205

[I 2022-11-03 01:22:12,084] Trial 3 finished with value: 2.391696088887076 and parameters: {'num_leaves': 95, 'max_depth': 6, 'learning_rate': 0.011615865989246453, 'n_estimators': 2850, 'class_weight': None, 'min_child_samples': 16, 'min_child_weight': 0.09767211400638387, 'reg_alpha': 0.6842330265121569, 'reg_lambda': 0.4401524937396013}. Best is trial 2 with value: 2.8290344637500597.

LGBM Regressor:  2.3912883596946313
LGBM Regressor:  2.8208975921341457
LGBM Regressor:  2.819047454832547
LGBM Regressor:  2.823712950746422
LGBM Regressor:  2.824712615346671


optuna.visualization.plot_param_importances(lgb_roc_study) # 파라미터 중요도 확인 그래프
optuna.visualization.plot_optimization_history(lgb_roc_study) # 최적화 과정 시각화


# CatBoost_r
def cat_r_objective(trial):
    param = {
      'iterations': trial.suggest_int('max_leaves', 10, 150),
      'depth': trial.suggest_int('depth', 5, 30),
      'learning_rate': trial.suggest_loguniform("learning_rate", 0.01, 0.1)
    }

    fold = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)
    cv_scores = []
    for train_idx, val_idx in fold.split(X_train, y_train):
        X_tr, X_val = X_train.iloc[train_idx], X_train.iloc[val_idx]
        y_tr, y_val = y_train.iloc[train_idx], y_train.iloc[val_idx]

        model = CatBoostRegressor(**param, grow_policy='Lossguide', random_state=SEED, silent=True)
        model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)], early_stopping_rounds=50, verbose=False)
        y_pred = model.predict(X_val)
        score = rmse(y_val, y_pred)
        print("CatBoost Regressor: ",score)
        cv_scores.append(score)

    return np.mean(cv_scores)

cat_roc_study = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=SEED))
cat_roc_study.optimize(cat_r_objective, n_trials=trials)

cat_roc_best = cat_roc_study.best_trial
cat_roc_best_params = cat_roc_best.params

model_scores['catb'] = cat_roc_best.value
model_params['catb'] = cat_roc_best_params

print('score: {0}, params: {1}'.format(cat_roc_best.value, cat_roc_best_params))

[I 2022-11-03 00:52:30,850] A new study created in memory with name: no-name-6cac2698-cd98-4bab-afd3-ec48373b3d98

CatBoost Regressor:  2.4203172118738667
CatBoost Regressor:  2.4207328436498545
CatBoost Regressor:  2.420617823483958
CatBoost Regressor:  2.420976727527521

[I 2022-11-03 00:52:41,672] Trial 0 finished with value: 2.420472945279792 and parameters: {'max_leaves': 50, 'depth': 22, 'learning_rate': 0.011332916993356774}. Best is trial 0 with value: 2.420472945279792.

CatBoost Regressor:  2.4197201198637592
CatBoost Regressor:  2.4166282889611397
CatBoost Regressor:  2.4173344271381714
CatBoost Regressor:  2.417109175499168
CatBoost Regressor:  2.4176134176147746

[I 2022-11-03 00:52:49,604] Trial 1 finished with value: 2.4169722807781904 and parameters: {'max_leaves': 36, 'depth': 22, 'learning_rate': 0.02380942778329892}. Best is trial 0 with value: 2.420472945279792.

CatBoost Regressor:  2.4161760946776982
CatBoost Regressor:  2.4047391117322454
CatBoost Regressor:  2.4048480737627487
CatBoost Regressor:  2.4053858685030503
CatBoost Regressor:  2.405599778036442

[I 2022-11-03 00:53:01,405] Trial 2 finished with value: 2.404811556900635 and parameters: {'max_leaves': 55, 'depth': 26, 'learning_rate': 0.06318029125054338}. Best is trial 0 with value: 2.420472945279792.

CatBoost Regressor:  2.4034849524686877
CatBoost Regressor:  2.4006167073523472
CatBoost Regressor:  2.399749302727278
CatBoost Regressor:  2.4017866162454533
CatBoost Regressor:  2.4011023925719965

[I 2022-11-03 00:53:21,293] Trial 3 finished with value: 2.400464959515697 and parameters: {'max_leaves': 105, 'depth': 30, 'learning_rate': 0.06855496343839224}. Best is trial 0 with value: 2.420472945279792.

CatBoost Regressor:  2.3990697786814104
CatBoost Regressor:  2.4115870742648338
CatBoost Regressor:  2.4119691521285005
CatBoost Regressor:  2.412281116261281
CatBoost Regressor:  2.411670050049987


optuna.visualization.plot_param_importances(cat_roc_study) # 파라미터 중요도 확인 그래프
optuna.visualization.plot_optimization_history(cat_roc_study) # 최적화 과정 시각화


reader = Reader(rating_scale=(1, 10))
train_data = Dataset.load_from_df(train_df[['uid', 'iid', 'ratings']], reader)
test_data = Dataset.load_from_df(test_df[['uid', 'iid', 'ratings']], reader)

fold = DatasetAutoFolds(df = train_df[['uid', 'iid', 'ratings']], reader = reader)
trainset = fold.build_full_trainset()

svd_wrapper=SVD(**model_params['svd'],random_state=SEED)
svd_wrapper.fit(trainset)

<surprise.prediction_algorithms.matrix_factorization.SVD at 0x7eff67de0310>


lgb_r_wrapper= LGBMRegressor(**model_params['lgb_r'] ,random_state=SEED)
lgb_r_wrapper.fit(X_train.select_dtypes(exclude='object'), y_train)

LGBMRegressor(class_weight='balanced', learning_rate=0.02750108543596555,
              max_depth=9, min_child_samples=31,
              min_child_weight=0.7574846332086538, n_estimators=57,
              num_leaves=10, random_state=8746, reg_alpha=0.3887691947863865,
              reg_lambda=0.19029119128000968)

LGBMRegressor(class_weight='balanced', learning_rate=0.02750108543596555,
              max_depth=9, min_child_samples=31,
              min_child_weight=0.7574846332086538, n_estimators=57,
              num_leaves=10, random_state=8746, reg_alpha=0.3887691947863865,
              reg_lambda=0.19029119128000968)


catb_wrapper=CatBoostRegressor(**model_params['catb'], grow_policy='Lossguide', random_state=SEED, silent=True)
catb_wrapper.fit(X_train.select_dtypes(exclude='object'), y_train, early_stopping_rounds=100)

<catboost.core.CatBoostRegressor at 0x7eff64387ca0>


from sklearn.ensemble import VotingRegressor
voting_model = VotingRegressor(estimators=[('lgbm', lgb_r_wrapper), ('catboost', catb_wrapper)])
voting_model.fit(X_train.select_dtypes(exclude='object'), y_train)
voting_pred = voting_model.predict(X_test.select_dtypes(exclude='object'))
score=rmse(voting_pred,y_test)
model_scores['ensemble']=score


model_scores

{'svd': 2.2877188756780997,
 'lgb_r': 2.8858343690076387,
 'catb': 2.429913367691559,
 'ensemble': 2.5314419537259227}

	uid	iid	ratings
0	41848	33783	9
1	6538	2644	8
2	11221	81890	10
3	16021	6142	9
4	4632	89180	3
...	...	...	...
61354	2752	14936	6
61355	8108	86566	7
61356	10534	18168	5
61357	40246	22419	8
61358	24097	15452	10

	user_id	age	location_country	uid
0	8	3.0	2.0	0.0
1	11400	4.0	2.0	2334.0
2	67544	3.0	2.0	1.0
3	85526	3.0	2.0	4789.0
4	96054	2.0	2.0	3936.0
...	...	...	...	...
68087	156948	4.0	0.0	44668.0
68088	169489	0.0	0.0	64230.0
68089	56072	0.0	0.0	49566.0
68090	64582	3.0	0.0	51901.0
68091	218092	0.0	0.0	58709.0

	age	location_country	year_of_publication	publisher	language	category
0	3.0	2.0	1997.0	375	0	8.000000
1	1.0	2.0	1986.0	451	0	8.000000
2	5.0	2.0	1994.0	679	0	5.000000
3	3.0	2.0	1994.0	440	0	7.000000
4	3.0	2.0	2000.0	14	0	7.224138
...	...	...	...	...	...	...
245431	3.0	3.0	2001.0	349	0	6.750000
245432	3.0	2.0	1983.0	345	0	6.000000
245433	4.0	2.0	1986.0	553	0	8.000000
245434	3.0	3.0	2001.0	426	3	8.000000
245435	5.0	3.0	2003.0	571	0	6.750000